{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#|default_exp data.mixed"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Mixed data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    ">DataLoader than can take data from multiple dataloaders with different types of data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#|export\n",
    "from packaging import version\n",
    "from torch.utils.data.dataloader import _MultiProcessingDataLoaderIter, _SingleProcessDataLoaderIter, _DatasetKind\n",
    "from fastai.data.load import _FakeLoader\n",
    "from fastai.tabular.core import *\n",
    "from tsai.imports import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#|export\n",
    "# This implementation of a mixed dataloader is based on a great implementation created by Zach Mueller in this fastai thread:\n",
    "# https://forums.fast.ai/t/combining-tabular-images-in-fastai2-and-should-work-with-almost-any-other-type/73197\n",
    "\n",
    "_loaders = (_MultiProcessingDataLoaderIter, _SingleProcessDataLoaderIter)\n",
    "\n",
    "\n",
    "class MixedDataLoader():\n",
    "    def __init__(self, *loaders, path='.', shuffle=False, device=None, bs=None):\n",
    "        \"Accepts any number of `DataLoader` and a device\"\n",
    "        self.path = path\n",
    "        device = ifnone(device, default_device())\n",
    "        self.device = device\n",
    "        self.c = None\n",
    "        self.d = None\n",
    "        self.bs = ifnone(bs, min([dl.bs for dl in loaders]))\n",
    "        for i, dl in enumerate(loaders):  # ensure all dls have the same bs\n",
    "            if hasattr(dl, 'vars'):\n",
    "                self.vars = dl.vars\n",
    "            if hasattr(dl, 'len'):\n",
    "                self.len = dl.len\n",
    "            if hasattr(dl, 'split_idxs'):\n",
    "                self.split_idxs = dl.split_idxs\n",
    "            dl.bs = self.bs\n",
    "            if i > 0 and hasattr(dl, 'get_idxs'):\n",
    "                dl.get_idxs = self.get_idxs_copy\n",
    "            dl.shuffle_fn = self.shuffle_fn\n",
    "            if self.c is None and hasattr(dl, \"c\"):\n",
    "                self.c = dl.c\n",
    "            if self.d is None and hasattr(dl, \"d\"):\n",
    "                self.d = dl.d\n",
    "            if i == 0:\n",
    "                self.dataset = dl.dataset\n",
    "            dl.to(device=device)\n",
    "        self.shuffle = shuffle\n",
    "        if not self.shuffle:\n",
    "            self.rng = np.arange(len(self.dataset)).tolist()\n",
    "        self.loaders = loaders\n",
    "        self.count = 0\n",
    "        self.fake_l = _FakeLoader(self, False, 0, 0, 0, \"\") if version.parse(fastai.__version__) >= version.parse(\"2.7\") \\\n",
    "            else _FakeLoader(self, False, 0, 0, 0) if version.parse(fastai.__version__) >= version.parse(\"2.1\") \\\n",
    "            else _FakeLoader(self, False, 0, 0)\n",
    "        if sum([len(dl.dataset) for dl in loaders]) > 0:\n",
    "            self._get_idxs()  # Do not apply on an empty dataset\n",
    "\n",
    "    def new(self, *args, **kwargs):\n",
    "        loaders = [dl.new(*args, **kwargs) for dl in self.loaders]\n",
    "        return type(self)(*loaders, path=self.path, device=self.device)\n",
    "\n",
    "#     def __len__(self): return len(self.loaders[0])\n",
    "    def __len__(self): return self.loaders[0].__len__()\n",
    "\n",
    "    def _get_vals(self, x):\n",
    "        \"Checks for duplicates in batches\"\n",
    "        idxs, new_x = [], []\n",
    "        for i, o in enumerate(x):\n",
    "            x[i] = o.cpu().numpy().flatten()\n",
    "        for idx, o in enumerate(x):\n",
    "            if not self._arrayisin(o, new_x):\n",
    "                idxs.append(idx)\n",
    "                new_x.append(o)\n",
    "        return idxs\n",
    "\n",
    "    def _get_idxs(self):\n",
    "        \"Get `x` and `y` indices for batches of data\"\n",
    "        self.n_inps = [dl.n_inp for dl in self.loaders]\n",
    "        self.x_idxs = self._split_idxs(self.n_inps)\n",
    "\n",
    "        # Identify duplicate targets\n",
    "        dl_dict = dict(zip(range(0, len(self.loaders)), self.n_inps))\n",
    "        outs = L([])\n",
    "        for key, n_inp in dl_dict.items():\n",
    "            b = next(iter(self.loaders[key]))\n",
    "            outs += L(b[n_inp:])\n",
    "        self.y_idxs = self._get_vals(outs)\n",
    "\n",
    "    def get_idxs_copy(self):\n",
    "        return self.loaders[0].get_idxs()\n",
    "\n",
    "    def __iter__(self):\n",
    "        z = zip(*[_loaders[i.fake_l.num_workers == 0](i.fake_l)\n",
    "                for i in self.loaders])\n",
    "        for b in z:\n",
    "            inps = []\n",
    "            outs = []\n",
    "            if self.device is not None:\n",
    "                b = to_device(b, self.device)\n",
    "            for batch, dl in zip(b, self.loaders):\n",
    "                if hasattr(dl, 'idxs'):\n",
    "                    self.idxs = dl.idxs\n",
    "                if hasattr(dl, 'input_idxs'):\n",
    "                    self.input_idxs = dl.input_idxs\n",
    "                batch = dl.after_batch(batch)\n",
    "                inps += batch[:dl.n_inp]\n",
    "                outs += batch[dl.n_inp:]\n",
    "            inps = tuple([tuple(L(inps)[idx]) if isinstance(idx, list) else inps[idx]\n",
    "                          for idx in self.x_idxs]) if len(self.x_idxs) > 1 else tuple(L(outs)[self.x_idxs][0])\n",
    "            # based on issue identified by @Wabinab https://github.com/timeseriesAI/tsai/pull/229\n",
    "            if len(self.y_idxs) == 0:\n",
    "                yield tuple((inps,))\n",
    "            outs = tuple(L(outs)[self.y_idxs]) if len(\n",
    "                self.y_idxs) > 1 else L(outs)[self.y_idxs][0]\n",
    "            yield inps, outs\n",
    "\n",
    "    def one_batch(self):\n",
    "        \"Grab one batch of data\"\n",
    "        with self.fake_l.no_multiproc():\n",
    "            res = first(self)\n",
    "        if hasattr(self, 'it'):\n",
    "            delattr(self, 'it')\n",
    "        return res\n",
    "\n",
    "    def shuffle_fn(self, idxs):\n",
    "        \"Generate the same idxs for all dls in each batch when shuffled\"\n",
    "        if self.count == 0:\n",
    "            self.shuffled_idxs = np.random.permutation(idxs)\n",
    "        # sort each batch\n",
    "        for i in range(len(self.shuffled_idxs)//self.bs + 1):\n",
    "            self.shuffled_idxs[i*self.bs:(i+1)*self.bs] = np.sort(\n",
    "                self.shuffled_idxs[i*self.bs:(i+1)*self.bs])\n",
    "        self.count += 1\n",
    "        if self.count == len(self.loaders):\n",
    "            self.count = 0\n",
    "        return self.shuffled_idxs\n",
    "\n",
    "    def show_batch(self):\n",
    "        \"Show a batch of data\"\n",
    "        for dl in self.loaders:\n",
    "            dl.show_batch()\n",
    "\n",
    "    def to(self, device): self.device = device\n",
    "\n",
    "    def _arrayisin(self, arr, arr_list):\n",
    "        \"Checks if `arr` is in `arr_list`\"\n",
    "        for a in arr_list:\n",
    "            if np.array_equal(arr, a):\n",
    "                return True\n",
    "        return False\n",
    "\n",
    "    def _split_idxs(self, a):\n",
    "        a_cum = np.array(a).cumsum().tolist()\n",
    "        b = np.arange(sum(a)).tolist()\n",
    "        start = 0\n",
    "        b_ = []\n",
    "        for i, idx in enumerate(range(len(a))):\n",
    "            end = a_cum[i]\n",
    "            b_.append(b[start:end] if end - start > 1 else b[start])\n",
    "            start = end\n",
    "        return b_\n",
    "\n",
    "\n",
    "class MixedDataLoaders(DataLoaders):\n",
    "    pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#|export\n",
    "\n",
    "def get_mixed_dls(*dls, device=None, shuffle_train=None, shuffle_valid=None, **kwargs):\n",
    "    _mixed_train_dls = []\n",
    "    _mixed_valid_dls = []\n",
    "    for dl in dls:\n",
    "        _mixed_train_dls.append(dl.train)\n",
    "        _mixed_valid_dls.append(dl.valid)\n",
    "        if shuffle_train is None: shuffle_train = dl.train.shuffle\n",
    "        if shuffle_valid is None: shuffle_valid = dl.valid.shuffle\n",
    "        if device is None: device = dl.train.device\n",
    "    mixed_train_dl = MixedDataLoader(*_mixed_train_dls, shuffle=shuffle_train, **kwargs)\n",
    "    mixed_valid_dl = MixedDataLoader(*_mixed_valid_dls, shuffle=shuffle_valid, **kwargs)\n",
    "    mixed_dls = MixedDataLoaders(mixed_train_dl, mixed_valid_dl, device=device)\n",
    "    return mixed_dls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tsai.data.tabular import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>workclass</th>\n",
       "      <th>education</th>\n",
       "      <th>marital-status</th>\n",
       "      <th>age</th>\n",
       "      <th>fnlwgt</th>\n",
       "      <th>salary</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Private</td>\n",
       "      <td>5th-6th</td>\n",
       "      <td>Separated</td>\n",
       "      <td>47.000000</td>\n",
       "      <td>225065.000159</td>\n",
       "      <td>&lt;50k</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Private</td>\n",
       "      <td>HS-grad</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>56.999999</td>\n",
       "      <td>84887.999356</td>\n",
       "      <td>&lt;50k</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Private</td>\n",
       "      <td>Assoc-voc</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>30.000000</td>\n",
       "      <td>176409.999275</td>\n",
       "      <td>&gt;=50k</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Private</td>\n",
       "      <td>Some-college</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>31.000000</td>\n",
       "      <td>232474.999969</td>\n",
       "      <td>&lt;50k</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Private</td>\n",
       "      <td>10th</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>26.000000</td>\n",
       "      <td>293984.002897</td>\n",
       "      <td>&lt;50k</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Private</td>\n",
       "      <td>HS-grad</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>54.000000</td>\n",
       "      <td>167770.000370</td>\n",
       "      <td>&gt;=50k</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Private</td>\n",
       "      <td>Bachelors</td>\n",
       "      <td>Never-married</td>\n",
       "      <td>25.000000</td>\n",
       "      <td>60357.998190</td>\n",
       "      <td>&lt;50k</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Local-gov</td>\n",
       "      <td>7th-8th</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>62.000000</td>\n",
       "      <td>203524.999993</td>\n",
       "      <td>&lt;50k</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Private</td>\n",
       "      <td>Some-college</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>36.000000</td>\n",
       "      <td>220510.999048</td>\n",
       "      <td>&lt;50k</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>State-gov</td>\n",
       "      <td>Doctorate</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>55.000000</td>\n",
       "      <td>120781.002923</td>\n",
       "      <td>&gt;=50k</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>education-num_na</th>\n",
       "      <th>education-num</th>\n",
       "      <th>salary</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>False</td>\n",
       "      <td>10.0</td>\n",
       "      <td>&lt;50k</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>False</td>\n",
       "      <td>9.0</td>\n",
       "      <td>&lt;50k</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>False</td>\n",
       "      <td>13.0</td>\n",
       "      <td>&lt;50k</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>False</td>\n",
       "      <td>13.0</td>\n",
       "      <td>&gt;=50k</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>False</td>\n",
       "      <td>10.0</td>\n",
       "      <td>&lt;50k</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>False</td>\n",
       "      <td>9.0</td>\n",
       "      <td>&lt;50k</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>False</td>\n",
       "      <td>9.0</td>\n",
       "      <td>&lt;50k</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>False</td>\n",
       "      <td>10.0</td>\n",
       "      <td>&lt;50k</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>False</td>\n",
       "      <td>14.0</td>\n",
       "      <td>&gt;=50k</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>False</td>\n",
       "      <td>9.0</td>\n",
       "      <td>&lt;50k</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "path = untar_data(URLs.ADULT_SAMPLE)\n",
    "df = pd.read_csv(path/'adult.csv')\n",
    "# df['salary'] = np.random.rand(len(df)) # uncomment to simulate a cont dependent variable\n",
    "target = 'salary'\n",
    "splits = RandomSplitter()(range_of(df))\n",
    "\n",
    "cat_names = ['workclass', 'education', 'marital-status']\n",
    "cont_names = ['age', 'fnlwgt']\n",
    "dls1 = get_tabular_dls(df, cat_names=cat_names, cont_names=cont_names, y_names=target, splits=splits, bs=512)\n",
    "dls1.show_batch()\n",
    "\n",
    "cat_names = None #['occupation', 'relationship', 'race']\n",
    "cont_names = ['education-num']\n",
    "dls2 = get_tabular_dls(df, cat_names=cat_names, cont_names=cont_names, y_names=target, splits=splits, bs=128)\n",
    "dls2.show_batch()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>workclass</th>\n",
       "      <th>education</th>\n",
       "      <th>marital-status</th>\n",
       "      <th>age</th>\n",
       "      <th>fnlwgt</th>\n",
       "      <th>salary</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Private</td>\n",
       "      <td>Masters</td>\n",
       "      <td>Divorced</td>\n",
       "      <td>44.000000</td>\n",
       "      <td>236746.000153</td>\n",
       "      <td>&gt;=50k</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Private</td>\n",
       "      <td>Bachelors</td>\n",
       "      <td>Never-married</td>\n",
       "      <td>22.000000</td>\n",
       "      <td>189950.000000</td>\n",
       "      <td>&lt;50k</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Private</td>\n",
       "      <td>HS-grad</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>56.999999</td>\n",
       "      <td>120302.001777</td>\n",
       "      <td>&lt;50k</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Private</td>\n",
       "      <td>HS-grad</td>\n",
       "      <td>Never-married</td>\n",
       "      <td>29.000000</td>\n",
       "      <td>131087.999775</td>\n",
       "      <td>&lt;50k</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Self-emp-not-inc</td>\n",
       "      <td>HS-grad</td>\n",
       "      <td>Never-married</td>\n",
       "      <td>35.000000</td>\n",
       "      <td>179171.000276</td>\n",
       "      <td>&lt;50k</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Self-emp-not-inc</td>\n",
       "      <td>HS-grad</td>\n",
       "      <td>Divorced</td>\n",
       "      <td>75.000001</td>\n",
       "      <td>242107.999406</td>\n",
       "      <td>&lt;50k</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Private</td>\n",
       "      <td>12th</td>\n",
       "      <td>Never-married</td>\n",
       "      <td>36.000000</td>\n",
       "      <td>137420.999182</td>\n",
       "      <td>&lt;50k</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Private</td>\n",
       "      <td>Doctorate</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>35.000000</td>\n",
       "      <td>189623.000011</td>\n",
       "      <td>&gt;=50k</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>education-num_na</th>\n",
       "      <th>education-num</th>\n",
       "      <th>salary</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>False</td>\n",
       "      <td>9.0</td>\n",
       "      <td>&lt;50k</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>False</td>\n",
       "      <td>9.0</td>\n",
       "      <td>&lt;50k</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>False</td>\n",
       "      <td>9.0</td>\n",
       "      <td>&lt;50k</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>False</td>\n",
       "      <td>9.0</td>\n",
       "      <td>&lt;50k</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>False</td>\n",
       "      <td>9.0</td>\n",
       "      <td>&lt;50k</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>False</td>\n",
       "      <td>10.0</td>\n",
       "      <td>&lt;50k</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>False</td>\n",
       "      <td>9.0</td>\n",
       "      <td>&lt;50k</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>False</td>\n",
       "      <td>10.0</td>\n",
       "      <td>&gt;=50k</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "dls = get_mixed_dls(dls1, dls2, bs=8)\n",
    "first(dls.train)\n",
    "first(dls.valid)\n",
    "torch.save(dls,'export/mixed_dls.pth')\n",
    "del dls\n",
    "dls = torch.load('export/mixed_dls.pth')\n",
    "dls.train.show_batch()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tsai.data.validation import TimeSplitter\n",
    "from tsai.data.core import TSRegression, get_ts_dls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[[5., 5., 5., 5., 5.],\n",
      "         [5., 5., 5., 5., 5.]],\n",
      "\n",
      "        [[0., 0., 0., 0., 0.],\n",
      "         [0., 0., 0., 0., 0.]],\n",
      "\n",
      "        [[4., 4., 4., 4., 4.],\n",
      "         [4., 4., 4., 4., 4.]],\n",
      "\n",
      "        [[3., 3., 3., 3., 3.],\n",
      "         [3., 3., 3., 3., 3.]]], device='mps:0') tensor([5., 0., 4., 3.], device='mps:0')\n",
      "tensor([[[6., 6., 6., 6., 6.],\n",
      "         [6., 6., 6., 6., 6.]],\n",
      "\n",
      "        [[1., 1., 1., 1., 1.],\n",
      "         [1., 1., 1., 1., 1.]],\n",
      "\n",
      "        [[2., 2., 2., 2., 2.],\n",
      "         [2., 2., 2., 2., 2.]],\n",
      "\n",
      "        [[7., 7., 7., 7., 7.],\n",
      "         [7., 7., 7., 7., 7.]]], device='mps:0') tensor([6., 1., 2., 7.], device='mps:0')\n"
     ]
    }
   ],
   "source": [
    "X = np.repeat(np.repeat(np.arange(16)[:, None, None], 2, 1), 5, 2).astype(float)\n",
    "y = np.concatenate([np.arange(len(X)//2)]*2)\n",
    "alphabet = np.array(list(string.ascii_lowercase))\n",
    "# y = alphabet[y]\n",
    "splits = TimeSplitter(.5, show_plot=False)(range_of(X))\n",
    "tfms = [None, TSRegression()]\n",
    "dls1 = get_ts_dls(X, y, splits=splits, tfms=tfms, bs=4)\n",
    "for xb, yb in iter(dls1.train):\n",
    "    print(xb.data, yb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cat1</th>\n",
       "      <th>cat2</th>\n",
       "      <th>cont</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "      <td>100.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>20</td>\n",
       "      <td>200.0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>30</td>\n",
       "      <td>300.0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>40</td>\n",
       "      <td>400.0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5</td>\n",
       "      <td>50</td>\n",
       "      <td>500.0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>6</td>\n",
       "      <td>60</td>\n",
       "      <td>600.0</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>7</td>\n",
       "      <td>70</td>\n",
       "      <td>700.0</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>8</td>\n",
       "      <td>80</td>\n",
       "      <td>800.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>9</td>\n",
       "      <td>90</td>\n",
       "      <td>900.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>10</td>\n",
       "      <td>100</td>\n",
       "      <td>1000.0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>11</td>\n",
       "      <td>110</td>\n",
       "      <td>1100.0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>12</td>\n",
       "      <td>120</td>\n",
       "      <td>1200.0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>13</td>\n",
       "      <td>130</td>\n",
       "      <td>1300.0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>14</td>\n",
       "      <td>140</td>\n",
       "      <td>1400.0</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>15</td>\n",
       "      <td>150</td>\n",
       "      <td>1500.0</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    cat1  cat2    cont  target\n",
       "0      0     0     0.0       0\n",
       "1      1    10   100.0       1\n",
       "2      2    20   200.0       2\n",
       "3      3    30   300.0       3\n",
       "4      4    40   400.0       4\n",
       "5      5    50   500.0       5\n",
       "6      6    60   600.0       6\n",
       "7      7    70   700.0       7\n",
       "8      8    80   800.0       0\n",
       "9      9    90   900.0       1\n",
       "10    10   100  1000.0       2\n",
       "11    11   110  1100.0       3\n",
       "12    12   120  1200.0       4\n",
       "13    13   130  1300.0       5\n",
       "14    14   140  1400.0       6\n",
       "15    15   150  1500.0       7"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[5, 5],\n",
      "        [5, 5],\n",
      "        [6, 6],\n",
      "        [4, 4]], device='mps:0') tensor([[400.],\n",
      "        [400.],\n",
      "        [500.],\n",
      "        [300.]], device='mps:0') tensor([[4],\n",
      "        [4],\n",
      "        [5],\n",
      "        [3]], device='mps:0', dtype=torch.int8)\n",
      "tensor([[4, 4],\n",
      "        [7, 7],\n",
      "        [2, 2],\n",
      "        [1, 1]], device='mps:0') tensor([[300.],\n",
      "        [600.],\n",
      "        [100.],\n",
      "        [  0.]], device='mps:0') tensor([[3],\n",
      "        [6],\n",
      "        [1],\n",
      "        [0]], device='mps:0', dtype=torch.int8)\n"
     ]
    }
   ],
   "source": [
    "data = np.repeat(np.arange(16)[:, None], 3, 1)*np.array([1, 10, 100])\n",
    "df = pd.DataFrame(data, columns=['cat1', 'cat2', 'cont'])\n",
    "df['cont'] = df['cont'].astype(float)\n",
    "df['target'] = y\n",
    "display(df)\n",
    "cat_names = ['cat1', 'cat2']\n",
    "cont_names = ['cont']\n",
    "target = 'target'\n",
    "dls2 = get_tabular_dls(df, procs=[Categorify, FillMissing, #Normalize\n",
    "                                 ], cat_names=cat_names, cont_names=cont_names, y_names=target, splits=splits, bs=4)\n",
    "for b in iter(dls2.train):\n",
    "    print(b[0], b[1], b[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bs = 8\n",
    "dls = get_mixed_dls(dls1, dls2, bs=bs)\n",
    "dl = dls.train\n",
    "xb, yb = dl.one_batch()\n",
    "test_eq(len(xb), 2)\n",
    "test_eq(len(xb[0]), bs)\n",
    "test_eq(len(xb[1]), 2)\n",
    "test_eq(len(xb[1][0]), bs)\n",
    "test_eq(len(xb[1][1]), bs)\n",
    "test_eq(xb[0].data[:, 0, 0].long(), xb[1][0][:, 0] - 1) # categorical data and ts are in synch\n",
    "test_eq(xb[0].data[:, 0, 0], (xb[1][1]/100).flatten()) # continuous data and ts are in synch\n",
    "test_eq(tensor(dl.input_idxs), yb.long().cpu())\n",
    "dl = dls.valid\n",
    "xb, yb = dl.one_batch()\n",
    "test_eq(tensor(y[dl.input_idxs]), yb.long().cpu())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[[0., 0., 0., 0., 0.],\n",
      "         [0., 0., 0., 0., 0.]],\n",
      "\n",
      "        [[1., 1., 1., 1., 1.],\n",
      "         [1., 1., 1., 1., 1.]],\n",
      "\n",
      "        [[2., 2., 2., 2., 2.],\n",
      "         [2., 2., 2., 2., 2.]],\n",
      "\n",
      "        [[4., 4., 4., 4., 4.],\n",
      "         [4., 4., 4., 4., 4.]]], device='mps:0') (tensor([[1, 1],\n",
      "        [2, 2],\n",
      "        [3, 3],\n",
      "        [5, 5]], device='mps:0'), tensor([[  0.],\n",
      "        [100.],\n",
      "        [200.],\n",
      "        [400.]], device='mps:0')) tensor([0., 1., 2., 4.], device='mps:0')\n",
      "tensor([[[3., 3., 3., 3., 3.],\n",
      "         [3., 3., 3., 3., 3.]],\n",
      "\n",
      "        [[5., 5., 5., 5., 5.],\n",
      "         [5., 5., 5., 5., 5.]],\n",
      "\n",
      "        [[6., 6., 6., 6., 6.],\n",
      "         [6., 6., 6., 6., 6.]],\n",
      "\n",
      "        [[7., 7., 7., 7., 7.],\n",
      "         [7., 7., 7., 7., 7.]]], device='mps:0') (tensor([[4, 4],\n",
      "        [6, 6],\n",
      "        [7, 7],\n",
      "        [8, 8]], device='mps:0'), tensor([[300.],\n",
      "        [500.],\n",
      "        [600.],\n",
      "        [700.]], device='mps:0')) tensor([3., 5., 6., 7.], device='mps:0')\n"
     ]
    }
   ],
   "source": [
    "bs = 4\n",
    "dls = get_mixed_dls(dls1, dls2, bs=bs)\n",
    "for xb, yb in iter(dls.train):\n",
    "    print(xb[0].data, xb[1], yb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": "IPython.notebook.save_checkpoint();",
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/Users/nacho/notebooks/tsai/nbs/015_data.mixed.ipynb saved at 2025-01-20 09:51:26\n",
      "Correct notebook to script conversion! 😃\n",
      "Monday 20/01/25 09:51:29 CET\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "                <audio  controls=\"controls\" autoplay=\"autoplay\">\n",
       "                    <source src=\"data:audio/wav;base64,UklGRvQHAABXQVZFZm10IBAAAAABAAEAECcAACBOAAACABAAZGF0YdAHAAAAAPF/iPh/gOoOon6w6ayCoR2ZeyfbjobxK+F2Hs0XjKc5i3DGvzaTlEaraE+zz5uLUl9f46fHpWJdxVSrnfmw8mYEScqUP70cb0Q8X41uysJ1si6Eh1jYzXp9IE2DzOYsftYRyoCY9dJ/8QICgIcEun8D9PmAaBPlfT7lq4MFIlh61tYPiCswIHX+yBaOqT1QbuW7qpVQSv9lu6+xnvRVSlyopAypbGBTUdSalrSTaUBFYpInwUpxOzhti5TOdndyKhCGrdwAfBUcXIJB69p+Vw1egB76+n9q/h6ADglbf4LvnIHfF/981ODThF4m8HiS0riJVjQ6c+/EOZCYQfJrGrhBmPVNMmNArLKhQlkXWYqhbaxXY8ZNHphLuBJsZUEckCTFVHMgNKGJytIDeSUmw4QN4Qx9pReTgb3vYX/TCBuApf75f+P5Y4CRDdN+B+tngk8c8nt03CKGqipgd13OhotwOC5x9MCAknFFcmlmtPmagFFFYOCo0qRzXMhVi57pryNmIEqJlRi8bm52PfuNM8k4dfQv+4cO12l6zCGdg3jl730uE/KAPvS+f0wEAoAsA89/XfXQgBESIn6S5luDtiC8eh/YmIfpLqt1OMp5jXg8/24MveqUNUnPZsqw0Z3yVDldnaUOqIZfXlKrm36zzWhjRhaT+r+ncHI5/otUzfd2uSt7hl/bqXtoHaCC6+mqfrAOeoDD+PJ/xf8RgLMHfH/b8GeBihZIfSXidoQSJWB52NM1iRkzz3MkxpKPbUCrbDu5d5fgTAxkSK3JoEhYD1p2omere2LZTuqYLbdWa49Cx5Dww7tyXDUnioXRkHhwJyKFvd/AfPoYy4Fl7j1/LQorgEr9/X89+0qAOAwAf13sJoL8Gkd8wt25hWIp3Heez/eKODfPcSPCzpFNRDVqf7UlmnNQKGHgqd+jgVvJVm2f265QZTpLS5byur1tpT6ajvrHq3Q2MXWIxtUCehoj8YMk5LB9hRQegeTypn+nBQWA0QHgf7f2q4C5EFt+5ucOg2YfHXtq2SSHpS0ydnTL4IxFO6pvNb4ulBdInWfcsfSc7VMmXpSmE6eeXmZThJxpsgRohEfOk86+AHCoOpOMFsx1dv8s6oYT2k17uR7ngpXod34IEJqAaPfnfyABCIBZBpl/NPI2gTQVjX134x2ExSPMeR7VtYjZMWJ0W8ftjkA/YW1durCWykvjZFKu4p9LVwVbZKNkqpxh6U+6mRC2mGq2Q3SRvsIgcpc2sIpD0Bp4uiiFhW3ecXxOGgaCDe0Vf4cLPoDv+/5/mfw1gN4KKX+17emBqBmYfBHfVYUZKFR44NBtiv41bHJUwx+RJkP1apu2VJlkTwli4qrwoo1ax1dToNCtemRSTBGXz7kJbdM/PY/Dxht0dTLziH7Ul3loJEiE0uJsfdsVTYGL8Yt/AgcMgHYA7X8S+IqAYA+QfjzpxIIVHnp7tdqzhmAstXaxzEqMETpScGC/dJP3Rmdo8LIZnOVSEF+Opxumsl1sVF+dVrE5Z6NIiZSkvVdv2zsqjdnK8HVDLlyHyNjuegogM4NA5z9+YRG9gA722H97AgOA/gSyf43zCIHdE899yuTIg3ciNXpm1jmImTDwdJPITI4RPhRugbvslbFKt2Vfr/6eTFb4W1WkY6m6YPdQjJr2tNZp3EQlko7BgXHRNz2LAc+gdwMq7IUf3R58ohtFgrbr6n7hDFWAlPr8f/T9I4CECU9/De+vgVQY5nxh4POEzybJeCTS5YnCNAZzhsRzkP1Bsmu4t4aYU07nYuerA6KWWcJYO6HHrKJjaE3Zl624UWz/QOOPjcWHc7QzdIk40yl5tCWjhIDhJX0xF4CBMvBsf10IF4Ac//Z/bPlsgAcOwn6S6n6CwxzUewLcRoYaKzV38M23i9o493CNwL6S1UUuaQe0QpvbUfdfiqglpcRccFU+nkWwambASUiVfLyqbg49xY2eyWh1hy/Sh37XjHpaIYKD7OUEfrgS5IC09MV/1gMBgKMDyH/n9N6AhhINfh7mdoMoIZt6r9fAh1cvfHXNya6N4DzDbqi8K5WWSYlmbbAdnkpV6FxJpWSo1V8DUmGb3rMRaQBG2JJgwN9wCDnNi8HNI3dKK1aG0dvHe/UciIJf6rt+Og5wgDn59X9P/xWAKQhxf2XweYH+FjB9suGVhIMlOnlo02GJhTOdc7vFyo/TQGxs2Li7lz9NwmPurBihnVi7WSWiwKvGYntOpJiOt5drKUKMkFnE8HLxNPmJ9NG4eP8mAYUv4Np8hhi3gdruSX+3CSWAwP38f8f6UoCuDPF+6Os8gnAbKnxQ3d2F0imydzDPKIuiN5lxu8EKkrFE82kftW2az1DbYImpMqTUW3FWIJ83r5hl2koJlla7+m0+PmSOZcjcdMgwS4g11iZ6qCLUg5jkxn0QFA6BWvOvfzEFBIBHAtp/Qfa3gC4RSH5y5yeD2B/8evnYS4cULgR2CMsUja47cG/QvW6UeEhXZ3+xP51GVNVdP6Zpp+1eDFM5nMeySWghR4+TNL85cD46YIyCzKJ2kCzEhoTabXtGHs+CCemJfpMPjoDe9+t/qQALgM8Gj3++8UaBqRV2fQTjO4Q3JKd5r9TgiEYyMHTxxiWPpz8jbfq585YpTJpk960xoKFXsVoTo7yq6GGMTw==\" type=\"audio/wav\" />\n",
       "                    Your browser does not support the audio element.\n",
       "                </audio>\n",
       "              "
      ],
      "text/plain": [
       "<IPython.lib.display.Audio object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#|eval: false\n",
    "#|hide\n",
    "from tsai.export import get_nb_name; nb_name = get_nb_name(locals())\n",
    "from tsai.imports import create_scripts; create_scripts(nb_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "python3",
   "language": "python",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
